﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Windows.Forms;

using SpiderCore.Utility;

namespace SpiderCore.Spiders
{
    /// <summary>
    /// 通用爬虫
    /// </summary>
    public abstract class Spider
    {

        /// <summary>
        /// WebBrowser组件
        /// </summary>
        public static WebBrowser WebBrowser { get; set; }
        /// <summary>
        /// WebBrowser组件访问延时秒
        /// </summary>
        public static int WebBrowserDelay { get; set; }
        /// <summary>
        /// 开始工作
        /// </summary>
        /// <param name="data"></param>
        public abstract void StartToWork(ToDoData data);

        /// <summary>
        /// 获取最终的请求Url地址
        /// </summary>
        /// <param name="sourceContent"></param>
        /// <param name="flag"></param>
        /// <returns></returns>
        protected abstract string GetFinalUrl(string sourceContent, string flag);
        /// <summary>
        /// 获取目标地址网页的源代码
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public string GetSourceHtml(string url, Encoding code)
        {
            if (!url.StartsWith("http"))
                return string.Empty;
            WebClient request = new WebClient();
            if (code != null)
                request.Encoding = code;
            try
            {
                return request.DownloadString(url);
            }
            catch (Exception ex)
            {
                if (ex.ToString().Contains("404"))
                {
                    CustomsLog.WriteExceptionErrorLog("获取目标地址网页的源代码异常", ex, "Url:" + url);
                    return string.Empty;
                }
                else if (ex.ToString().Contains("403"))
                {
                    throw;
                }
                throw;
            }
           
        }
        /// <summary>
        /// 通过浏览器组件获取最终的HTML
        /// </summary>
        /// <param name="url"></param>
        public string GetFinalHtmlByWebBrowser(string url)
        {
            if (string.IsNullOrWhiteSpace(url)) return null;
            WebBrowser.Navigate(url);
            DateTime timestart = DateTime.Now;
            //先要等待主页加载完毕
            var comp = false;
            WebBrowser.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler((sender, e) =>
            {
                comp = true;
            });
            while (!comp)
            {
                if (DateTime.Now - timestart > new TimeSpan(0, 0, 0, 30))//最大超时30秒
                {
                    CustomsLog.WriteInfoLog("访问URL超时", "Url:" + url);
                    return null;
                }
                Application.DoEvents();
            }
            while (WebBrowser.ReadyState != WebBrowserReadyState.Complete)
            {
                if (DateTime.Now - timestart > new TimeSpan(0, 0, 0, 30))//最大超时30秒
                {
                    CustomsLog.WriteInfoLog("访问URL超时", "Url:" + url);
                    return null;
                }
                Application.DoEvents();
            }
            SpiderHelper.DelayTime((WebBrowserDelay == 0 ? 10 : WebBrowserDelay));
            var htmldocument = (mshtml.HTMLDocument)WebBrowser.Document.DomDocument;
            return htmldocument.documentElement.outerHTML; ;
        }
        /// <summary>
        /// 
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public string GetWebContentWithCookie(string url, CookieContainer cookie)
        {
            HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(url);
            myRequest.Method = "GET";
            myRequest.ContentType = "application/x-www-form-urlencoded";
            myRequest.CookieContainer = cookie;
            HttpWebResponse myResponse = (HttpWebResponse)myRequest.GetResponse();
            StreamReader reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
            return reader.ReadToEnd();
        }

        /// <summary>
        /// 提取网页中的链接
        /// </summary>
        /// <param name="strHtml"></param>
        /// <param name="domain">匹配地址域名</param>
        /// <param name="filetype">匹配文件后缀名</param>
        /// <param name="lenth">最大地址长度，避免陷入困境</param>
        /// <returns></returns>
        public List<string> GetLink(string strHtml, string domain = null, string filetype = null,int lenth=300)
        {

            string RegexStr = "(?<=href[\\s\\r]*=[\\s\\r]*[\"\'])([^\"\'\\s\\r>]*)(?<=[\"\'\\s\\r]{0,1})";
            List<string> result = new List<string>();
            Regex reg = new Regex(RegexStr, RegexOptions.IgnoreCase);
            MatchCollection mc = reg.Matches(strHtml);
            if (mc.Count > 0)
            {
                foreach (Match m in mc)
                {
                    string v = m.Groups[1].Value;
                    if (!string.IsNullOrWhiteSpace(domain) && !v.Contains(domain))
                        continue;
                    if (!string.IsNullOrWhiteSpace(filetype) && !v.EndsWith(filetype))
                        continue;
                    if(v.Length>300)
                        continue;
                    result.Add(m.Groups[1].Value); ;
                }
                return result.Distinct().ToList();
            }
            return result;
        }

    }
}
